
package com.ordern.classification.classifiers;

/**
 * @author sghosh
 *
 */

/*
  Some requirements and TODOs for business rule class as well as applet:

    - the user should be able to type in a text and the system will classify
      the text.

      One way of doing this is to have to TextAreas:
      * one for spam messages
      * one for no spams


    - there should be at least two classes (e.g. spam/no spam).

    - the user should be able to define his/her own instances, including
      defining own classes. I.e. just the classes "0", "1" will not do
      (at least not in a more elaborated version). 

    - there should be a possibility to select different examples
      (probably via the <PARAMETER> tag in the applet HTML page).
      This is mainly for showing some different applications. (There are more
      applications than spam classification!)

    - I must find at least one example besides spam/no spam which are easy to do

    BUGS:
    - a boring bug is that some classifiers don't work if the test string
      contains a word not included in the original classifier. Which is not
      that strange if one think about it; but still.
      SOLUTION: put all words in the original model into a a Hash/Set and
      for the test string just check the words contained in the original model.
      But then comes the problem of deciding which class to choose if there is
      no original word in the test string!

      Is there any way of getting the default class from a classifier?

      FIX: I just test the words (model words) that's in the modeling 
      classification.

*/


import java.io.File;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;

import weka.classifiers.Classifier;
import weka.classifiers.Evaluation;
import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.stemmers.SnowballStemmer;
import weka.core.stemmers.Stemmer;
import weka.filters.unsupervised.attribute.StringToWordVector;

import com.ordern.classification.CustomClassifier;
import com.ordern.classification.Domain;
import com.ordern.classification.DomainType;


public class DCClassifier implements CustomClassifier, Serializable {

    private String[]   inputText       = null;
    private String[]   inputClasses    = null;
    private String     classifierType     = null;

    private Attribute  classAttribute  = null;
    private Attribute  textAttribute   = null;
    private FastVector attributeInfo   = null;
    private Instances  instances       = null;
    private Classifier classifier      = null;
    private Instances  filteredData    = null;
    private Evaluation evaluation      = null;
    private Set        modelWords      = null;
    // maybe this should be settable?
    private String     delimitersStringToWordVector = "\\s.,:'\\\"()?!";

    /////////////////////////////////////
    
    private Classifier metaTagClassifier      = null;
    private Classifier titleClassifier      = null;
    private Classifier bodyClassifier      = null;
    
    private Attribute metaDataClassAttribute = null;
    private Attribute titleClassAttribute = null;
    private Attribute bodyClassAttribute = null;
    private Attribute metaDataTextAttribute = null;
    private Attribute titleTextAttribute = null;
    private Attribute bodyTextAttribute = null;
    
    private Instances  metaDataInstances       = null;
    private Instances  titleInstances       = null;
    private Instances  bodyInstances       = null;
    
    private Instances  metaDataFiltered       = null;
    private Instances  titleFiltered       = null;
    private Instances  bodyFiltered       = null;
    
    private File stopwordList = null;
    //
    // main, mainly for testing
    //
    public static void main(String args[]) {
/*
        // String classifierType = "weka.classifiers.bayes.NaiveBayes";
        //String classifierType = "weka.classifiers.lazy.IBk";
         String classifierType = "weka.classifiers.trees.J48";
         //String classifierType = "weka.classifiers.functions.SMO";

        if (args.length > 0) {
        	classifierType = args[0];
        } 

        String[] inputText =  {"hey, buy this from me!", "do you want to buy?", "I have a party tonight!", "today it is a nice weather","you are best","I have a horse","you are my friend","buy, buy, buy!","it is spring in the air", "do you want to come?"};

        
        String[] inputClasses = {"spam","spam","no spam","no spam","spam","no spam","no spam","spam","no spam","no spam"};
        
        String[] testText = {"you want to buy from me?","usually I run in stairs", "buy it now!","buy, buy, buy!","you are the best, buy!","it is spring in the air"};
        
*/
    	SnowballStemmer ss =	new SnowballStemmer();
    	
    	System.out.println(ss.stem("wordings"));
    	
    } // end main


    //
    DCClassifier(String[] inputText, String[] inputClasses, FastVector attributeInfo, Attribute textAttribute, Attribute classAttribute, String classString) {
        this.inputText      = inputText;
        this.inputClasses   = inputClasses;
        this.classifierType    = classString;
        this.attributeInfo  = attributeInfo;
        this.textAttribute  = textAttribute;
        this.classAttribute = classAttribute;     

    }

    public DCClassifier (String classifierType, String stopWordFile) {
        this.classifierType    = classifierType;
        
        System.out.println("initializing DCClassifier");
        
        try {
        	
        	//this.stopwordList = new File(stopWordFile);
        	
			this.metaTagClassifier = Classifier.forName(classifierType,null);
	        this.titleClassifier = Classifier.forName(classifierType,null);
	        this.bodyClassifier = Classifier.forName(classifierType,null);
        } catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
    }

private void initialize(List<Domain> domains) {
    int size = domains.size();
    try {
    // Create vector of attributes.
    FastVector metaAttributes = new FastVector(2);
    
    FastVector titleAttributes = new FastVector(2);
    
    FastVector bodyAttributes = new FastVector(2);

    // Add attribute for holding meta tags.
    metaDataTextAttribute = new Attribute("Meta", (FastVector) null);
    metaAttributes.addElement(metaDataTextAttribute);
    
    // Add attribute for holding titles.
    this.titleTextAttribute = new Attribute("Title", (FastVector) null);
    titleAttributes.addElement(this.titleTextAttribute);

    // Add attribute for holding body.
    this.bodyTextAttribute = new Attribute("Body", (FastVector) null);
    bodyAttributes.addElement(this.bodyTextAttribute);
    
    // Add class attribute.
    FastVector classValues = new FastVector(3);
    classValues.addElement(DomainType.PERSONAL.name());
    classValues.addElement(DomainType.NOT_PERSONAL.name());
    classValues.addElement(DomainType.NOT_DETERMINED.name());
    
    this.metaDataClassAttribute = new Attribute("class", classValues);
    this.titleClassAttribute = new Attribute("class", classValues);
    this.bodyClassAttribute = new Attribute("class", classValues);
    
    metaAttributes.addElement(metaDataClassAttribute);
    titleAttributes.addElement(this.titleClassAttribute);
    bodyAttributes.addElement(this.bodyClassAttribute);

    // Create dataset with initial capacity of 100, and set index of class.
    metaDataInstances = new Instances("meta data set", metaAttributes, size);
    titleInstances = new Instances("title data set", titleAttributes, size);
    bodyInstances = new Instances("body data set", bodyAttributes, size);
    // m_Data.setClassIndex(m_Data.numAttributes() - 1);
    
    // set which attribute is the class attribute
    metaDataInstances.setClass(metaDataClassAttribute);
    titleInstances.setClass(titleClassAttribute);
    bodyInstances.setClass(bodyClassAttribute);
    
    System.out.println("initializing instances");
    
    } catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
}
    
    //
    // take instances in normal format (strings) and convert to Sparse format
    //
    public Instances filterText(Instances theseInstances) {

        StringToWordVector filter = null;
        // default values according to Java Doc:
        int wordsToKeep = 1000;

        Instances filtered = null;

        try {

            filter = new StringToWordVector(wordsToKeep);
            // we ignore this for now...
            // filter.setDelimiters(delimitersStringToWordVector);
            filter.setOutputWordCounts(true);
            filter.setSelectedRange("1");
           // filter.setStopwords(this.stopwordList);
            filter.setStemmer(new SnowballStemmer());
            
            
            System.out.println(" +++++++++++++++++ " +filter.setInputFormat(theseInstances));
            
            filtered = weka.filters.Filter.useFilter(theseInstances,filter);
             System.out.println("filtered:\n" + filtered);
            
        } catch (Exception e) {
            e.printStackTrace();
        }
        
        return filtered;
        
    } // end filterText


    //
    // information about classifier and evaluation
    //
    public static StringBuffer printClassifierAndEvaluation(Classifier thisClassifier, Evaluation thisEvaluation) {

        StringBuffer result = new StringBuffer();

        try {
            result.append("\n\nINFORMATION ABOUT THE CLASSIFIER AND EVALUATION:\n");
            result.append("\nclassifier.toString():\n" + thisClassifier.toString() + "\n");
            result.append("\nevaluation.toSummaryString(title, false):\n" + thisEvaluation.toSummaryString("Summary",false)  + "\n");
            result.append("\nevaluation.toMatrixString():\n" + thisEvaluation.toMatrixString()  + "\n");
            result.append("\nevaluation.toClassDetailsString():\n" + thisEvaluation.toClassDetailsString("Details")  + "\n");
            result.append("\nevaluation.toCumulativeMarginDistribution:\n" + thisEvaluation.toCumulativeMarginDistributionString()  + "\n");
        } catch (Exception e) {
            e.printStackTrace();
            result.append("\nException (sorry!):\n" + e.toString());
        }

        return result;

    } // end printClassifierAndEvaluation

    
	public void train ( List<Domain> boosterDomains, List<Domain> domains ) {
		
		 try {
			 initialize(domains); 
			 
			 System.out.println("pupulating Training data ...");
			 
			 for(Domain d : domains ) {
				      // System.out.println("pupulating Training data for domain ... " + d.getName() + " ::: meta [" +d.getMetaContent()+"] " +
				      // 		" Title [" + d.getTitle() + "] Body []");
					// populate meta tag information

		            if(null != d.getMetaContent() && d.getMetaContent().length() > 0 ) {
		    			Instance inst = new Instance(2);
		                inst.setValue(this.metaDataTextAttribute,d.getMetaContent());
		                inst.setValue(this.metaDataClassAttribute, d.getDomainType().name());
		                this.metaDataInstances.add(inst);
		            }

		            if(null != d.getTitle() && d.getTitle().length() > 0 ) {
		    			Instance inst = new Instance(2);
		                inst.setValue(this.titleTextAttribute,d.getTitle());
		                inst.setValue(this.titleClassAttribute, d.getDomainType().name());
		                this.titleInstances.add(inst);
		            }
		            
		            if(null != d.getBody() && d.getBody().length() > 0 ) {
		    			Instance inst = new Instance(2);
		                inst.setValue(this.bodyTextAttribute,d.getBody());
		                inst.setValue(this.bodyClassAttribute, d.getDomainType().name());
		                this.bodyInstances.add(inst);
		            }
		            
				}
				
			 
			 System.out.println("filtering Training data ...");
			 
		        // make filtered SparseData
		        this.metaDataFiltered = filterText(this.metaDataInstances);
		        this.titleFiltered = filterText(this.titleInstances);
		        this.bodyFiltered = filterText(this.bodyInstances);
		        
		        //bUILDING CLASSIFIER
		        System.out.println("building Classifiers ...");
	            this.metaTagClassifier.buildClassifier(this.metaDataFiltered);
	            this.titleClassifier.buildClassifier(this.titleFiltered);
	           // this.bodyClassifier.buildClassifier(this.bodyFiltered);
	            //evaluation = new Evaluation(filteredData);
	            //evaluation.evaluateModel(classifier, filteredData);
	           // weka.core.SerializationHelper.write("C:/work/Data Contest/summer2011datacontest-testdata/testdata/bodyclassifier.model", this.bodyClassifier);
	            System.out.println("Done with Training ...");
		        
		 } catch (Exception e) {
			 e.printStackTrace();
		 }

	}

	
	
	public void test(List<Domain> domains){
		 try {
			 List<Domain> meta = new ArrayList<Domain>();
			 List<Domain> title = new ArrayList<Domain>();
			 List<Domain> body = new ArrayList<Domain>();
			 Stemmer stm = new SnowballStemmer();
			 
			 int wordsToKeep = 1000;
			 // Make separate little test set so that message
			 // does not get added to string attribute in m_Data.
			 Instances metaTestset = this.metaDataInstances.stringFreeStructure();
			 StringToWordVector metaFilter = new StringToWordVector(wordsToKeep);  
		      // Initialize filter and tell it about the input format
			 metaFilter.setOutputWordCounts(true);
			 metaFilter.setSelectedRange("1");
			 //metaFilter.setStopwords(this.stopwordList);
			 metaFilter.setStemmer(stm);
			 metaFilter.setInputFormat(this.metaDataInstances);
			 
			 
			 Instances titleTestset = this.titleInstances.stringFreeStructure();
			 StringToWordVector titleFilter = new StringToWordVector(wordsToKeep);  
		      // Initialize filter and tell it about the input format
			 titleFilter.setOutputWordCounts(true);
			 titleFilter.setSelectedRange("1");
			 //titleFilter.setStopwords(this.stopwordList);
			 titleFilter.setStemmer(stm);
			 System.out.println("===============" + titleFilter.setInputFormat(this.titleInstances));
			// titleFilter
			 
			 Instances bodyTestset = this.bodyInstances.stringFreeStructure();
			 StringToWordVector bodyFilter = new StringToWordVector(wordsToKeep);  
		      // Initialize filter and tell it about the input format
			 bodyFilter.setOutputWordCounts(true);
			 bodyFilter.setSelectedRange("1");
			 //bodyFilter.setStopwords(this.stopwordList);
			 bodyFilter.setStemmer(stm);
			 bodyFilter.setInputFormat(this.bodyInstances);
			 
			 System.out.println("Classification started..... ");
			  for(Domain d : domains ) {
				  
				  double predicted = 0;
				  
				  System.out.println("pupulating test data for domain ... " + d.getName() + " ::: meta [" +d.getMetaContent()+"] " +
				       		" Title [" + d.getTitle() + "] Body []");
					// populate meta tag information
				  
		            if(null != d.getMetaContent() && d.getMetaContent().length() > 0 ) {
		    			Instance inst = new Instance(2);
		    		    // Set value for message attribute
		    		    Attribute messageAtt = metaTestset.attribute("Meta");
		    		    inst.setValue(messageAtt, messageAtt.addStringValue(d.getMetaContent()));

		    		    // Give instance access to attribute information from the dataset.
		    		    inst.setDataset(metaTestset);
		    		    
		    		    metaTestset.add(inst);
		    		    
		    		    metaFilter.input(inst);
		    		    meta.add(d);
		    		    
		            } 
		            else if(null != d.getBody() && d.getBody().length() > 0 ) {
		    			Instance inst = new Instance(2);
		    			
		    		    // Set value for message attribute
		    		    Attribute messageAtt = bodyTestset.attribute("Body");
		    		    inst.setValue(messageAtt, messageAtt.addStringValue(d.getBody()));
		    		    System.out.println("TESTING: " + inst.stringValue( bodyTestset.attribute("Body")));

		    		    // Give instance access to attribute information from the dataset.
		    		    inst.setDataset(bodyTestset);
		    		    
		    		    bodyTestset.add(inst);
		    		    body.add(d);
		            } 
		            else if (null != d.getTitle() && d.getTitle().length() > 0 ) {
		    			Instance inst = new Instance(2);
		    			
		    		    // Set value for message attribute
		    		    Attribute messageAtt = titleTestset.attribute("Title");
		    		    inst.setValue(messageAtt, messageAtt.addStringValue(d.getTitle()));
		    		    System.out.println("TESTING: " + inst.stringValue( titleTestset.attribute("Title")));

		    		    // Give instance access to attribute information from the dataset.
		    		    inst.setDataset(titleTestset);
		    		    
		    		    titleTestset.add(inst);
		    		    title.add(d);
		            }

				}
			  
			  Classifier desrializedBodyclssifier = (Classifier) weka.core.SerializationHelper.read("C:/work/Data Contest/summer2011datacontest-testdata/testdata/bodyclassifier.model");
			  
			  
			  
			  classification(meta, metaTestset, metaFilter,this.metaDataInstances, this.metaTagClassifier );
			  classification(title, titleTestset, titleFilter,this.titleInstances, this.titleClassifier );
			  classification(body,bodyTestset,bodyFilter,this.bodyInstances,  desrializedBodyclssifier);
		        
		 } catch (Exception e) {
			 e.printStackTrace();
		 }
	}


	private void classification(List<Domain> list,
			Instances testset, StringToWordVector filter, Instances insts, Classifier classf)
			throws Exception {
		int limit = testset.numInstances();
		filter.setInputFormat(testset);
		  Instances filteredInstance = weka.filters.Filter.useFilter(testset,filter);
		  for(int i =0; i < limit; i++ ) {
			    // Get index of predicted class value.
			    Instance instance = filteredInstance.instance(i);
				double predicted = classf.classifyInstance(instance);

			    Domain d = list.get(i);
			    
			    System.out.println(predicted);
			    
			    // Output class value.

			    
			    switch ((int)predicted) {
				case 0:
					d.setDomainType(DomainType.PERSONAL);
					break;
				case 1:
					d.setDomainType(DomainType.NOT_PERSONAL);
					break;
				default:
					d.setDomainType(DomainType.NOT_DETERMINED);
					break;
				}
			  
	    		    System.out.println("Domain ::'" + d.getName()+"' by body classified as : " +
	    		    		insts.classAttribute().value((int) predicted) + " " +
	    		    		d.getDomainType());
//	    		    System.out.println("CORRELATING: OUR Domain: " + d.getTitle() + "\n AND From CLASSIFIER: " + 
//	    		    		instance.stringValue( titleTestset.attribute("Title")));
//	    		    for (int j = 0; j < instance.numAttributes(); j++) {
//						System.out.println("Value NEW: "+ j +" - " + instance.value(j));
//					}
		  }
	}
	
	private String prepareMetaContent(Domain d) {
		
		return d.getMetaContent();
	}

	
} // end class TextClassifier


